package com.smoed.common.utilities.html;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.URLDecoder;
import java.net.UnknownHostException;
import java.util.ArrayList;

import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.EncodingChangeException;
import org.htmlparser.util.ParserException;
import org.htmlparser.visitors.ObjectFindingVisitor;

import com.smoed.common.logging.Log;

public class HTTPPageGetter {

    private static final String[] CONTAINS_URL_BLOCK_LIST = new String[] {
        "(?i)^javascript:.*"
    };

    // Not exhaustive, but it doesn't really matter because we will always
    // check the content type anyway.  However, if a url matches a block
    // list, it won't get sent back to the server.
    private static final String[] ENDS_WITH_URL_BLOCK_LIST = new String[] {
        ".pdf", ".zip", ".jpg", ".jpeg", ".png", ".gif", ".exe", ".o", ".tar.gz", ".rar"
    };

    public static final int MAX_URL_LENGTH = 2083;  // this is the maximum length for Internet Explorer
    public final static String UTF8_ENCODING = "UTF-8";
    private final static String TEXT_TYPE = "text";
    private static final String USER_AGENT = "User-agent";
    private static final String MOZILLA_VERSION = "Mozilla/4.76";

    private static final int URL_CONNECTION_TIMEOUT = 30000; // milliseconds
    private static final double STRING_LENGTH_PER_MILLISECOND_TO_KILOBYTE_PER_SECOND_CONVERSION_FACTOR = 7.8125;

    public PageData getPageData(String urlStr) {
        PageData pageData = null;
        try {
            urlStr = URLDecoder.decode(urlStr, UTF8_ENCODING);
            try {
                final URL url = new URL(urlStr);
                final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
                if (connection != null) {
                    try {
                        connection.setConnectTimeout(URL_CONNECTION_TIMEOUT);
                        connection.setReadTimeout(URL_CONNECTION_TIMEOUT);
                        connection.setRequestProperty(USER_AGENT, MOZILLA_VERSION);

                        final String contentType = connection.getContentType();
                        final String contentEncoding = connection.getContentEncoding();
                        long lastModified = connection.getLastModified();
                        connection.getResponseCode();
                        lastModified = (lastModified <= 0) ? System.currentTimeMillis() : lastModified;

                        final long date = connection.getDate();

                        // If url is content type containing the word "text", then retrieve it
                        if (contentType.contains(TEXT_TYPE)) {

                            final long start = System.currentTimeMillis();
                            BufferedReader in = null;
                            final StringBuilder src = new StringBuilder();

                            try {
                                in = new BufferedReader(new InputStreamReader(connection.getInputStream()));

                                String line;
                                while ((line = in.readLine()) != null) {
                                    src.append(line);
                                    src.append('\n');
                                }
                            } finally {
                                if (in != null) {
                                    in.close();
                                }
                            }
                            final String connectionUrl = connection.getURL().toString();
                            double retrievalTime = (System.currentTimeMillis() - start);
                            // Convert to kB/s
                            if (retrievalTime > 0 && src.length() > 0) {
                                retrievalTime = (src.length() / retrievalTime) * STRING_LENGTH_PER_MILLISECOND_TO_KILOBYTE_PER_SECOND_CONVERSION_FACTOR;
                            } else {
                                retrievalTime = -1;
                            }

                            final HtmlCleaner cleaner = new HtmlCleaner();
                            final TagNode rootNode = cleaner.clean(src.toString());
                            final String cleanHtml = cleaner.getInnerHtml(rootNode);

                            // Get links from page
                            final ArrayList<URL> links = extractLinks(connectionUrl, cleanHtml);

                            pageData = new PageData(urlStr, links, cleanHtml, contentType, contentEncoding, date, lastModified, retrievalTime);
                        }
                    } finally {
                        if (connection != null) {
                            connection.disconnect();
                        }
                    }
                } else {
                    Log.warn("Could not create connection for url: " + urlStr);
                }

            } catch (NullPointerException e) {
                Log.warn("Null pointer exception while connecting to: " + urlStr);
            }  catch(IllegalArgumentException e) {
                Log.warn("Illegal argument exception: " + urlStr);
            } catch (SocketTimeoutException e) {
                Log.warn("No response from url server: " + urlStr);
            } catch (UnknownHostException e) {
                Log.warn("Unknown host, page not found: " + urlStr);
            } catch (MalformedURLException e) {
                e.printStackTrace();
                Log.warn("Malformed URL: " + urlStr);
            }
        } catch (IOException e) {
            Log.warn("HTTP server returned an error code for url: " + urlStr);
        }

        return pageData;
    }

    private ArrayList<URL> extractLinks(String connectionUrl, String html) throws UnsupportedEncodingException {

        ArrayList<URL> links = new ArrayList<URL>();
        final Parser mParser = Parser.createParser(html, UTF8_ENCODING);

        try {
            //mParser.setConnection(httpUrlConn);
            mParser.setURL(connectionUrl);
            ObjectFindingVisitor visitor = new ObjectFindingVisitor(LinkTag.class);
            mParser.visitAllNodesWith(visitor);

            for (Node node : visitor.getTags()) {
                try {
                    links.add(new URL(((LinkTag)node).getLink()));
                } catch (MalformedURLException e) {
                }
            }

        } catch (EncodingChangeException e) {
        } catch (ParserException e) {
            //e.printStackTrace();
            Log.error("Could not extract links from html:\n");
        }

        return links;
    }


    public static boolean isBlockedUrl(String url) {

        final String lowerCaseUrl = url.toLowerCase();

        for (String blocker : CONTAINS_URL_BLOCK_LIST) {
            if (lowerCaseUrl.matches(blocker)) {
                return true;
            }
        }

        for (String blocker : ENDS_WITH_URL_BLOCK_LIST) {
            if (lowerCaseUrl.endsWith(blocker)) {
                return true;
            }
        }

        return false;
    }
}
